# -*- coding:utf-8 -*-
# @Time     : 2021/3/31 14:23


'''
    采集百度文库12个模块
'''
from redis import Redis
from pymongo import MongoClient
import requests
import random
from hashlib import md5
from concurrent.futures import ProcessPoolExecutor
import time

headers = {
    'Cookie': "__yjs_duid=1_d81926bbdf4cc5fdd154097e5a8654ea1615442955251; BAIDUID_BFESS=8CBF49D388137BF837BD67EEFE053448:FG=1; BAIDUID=84F0F7DCDAC3AA6D062BDFA50848EF98:FG=1; BIDUPSID=84F0F7DCDAC3AA6D062BDFA50848EF98; PSTM=1615771772; _click_param_reader_query_ab=-1; _click_param_pc_rec_doc_2017_testid=6; EDUAISOURCE=https%3A%2F%2Fcn.bing.com%2F; BDUSS=FtUmJVM1JWc2p4TDRuSFh3TWpKekdpQTlIMThBNXhaZThGalhXdWZubWhYNHBnRVFBQUFBJCQAAAAAAAAAAAEAAADhUWJ12Lywos3Y2LwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKHSYmCh0mJgQk; BDUSS_BFESS=FtUmJVM1JWc2p4TDRuSFh3TWpKekdpQTlIMThBNXhaZThGalhXdWZubWhYNHBnRVFBQUFBJCQAAAAAAAAAAAEAAADhUWJ12Lywos3Y2LwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKHSYmCh0mJgQk; kunlunFlag=1; isJiaoyuVip=1; userFirstTime=true; BCLID=6550554706366521910; BDSFRCVID=VRFOJeC626Ch_WQejK9dMRS5TMTpEHTTH6-Jfn6RbfyywV3sZ8KoEG0P8f8g0Ku-KVl7ogKKLgOTHULF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF=tbAf_K0MJCvbfP0kD5KbMb8t52T22-us3Dtj2hcH0KLKDMONKfoxbf0OjN7y2lRltCTiah6gaMb1M66_bT565Mobbqj7B4nO0DQUQl5TtUtWeCnTDMRhqJ_lQxjyKMniBIv9-pnGBpQrh459XP68bTkA5bjZKxtq3mkjbPbDfn02eCKujTL5D5JXjGRf-b-X-TT8LRj85-5ojRIk-PnVepKwQ4nZKxtqtJcmBU76LR58OITgXq56jq0I3h34Bp3nWnFLaJQ8JfoDe4Tw5pD5KpTbyUn405OTfIFO0KJcb66vSR46hPJvyUAsXnO7Lx7lXbrtXp7_2J0WStbKy4oTjxL1Db3JKjvMtgDtVJO-KKC-MKPljM5; BCLID_BFESS=6550554706366521910; BDSFRCVID_BFESS=VRFOJeC626Ch_WQejK9dMRS5TMTpEHTTH6-Jfn6RbfyywV3sZ8KoEG0P8f8g0Ku-KVl7ogKKLgOTHULF_2uxOjjg8UtVJeC6EG0Ptf8g0M5; H_BDCLCKID_SF_BFESS=tbAf_K0MJCvbfP0kD5KbMb8t52T22-us3Dtj2hcH0KLKDMONKfoxbf0OjN7y2lRltCTiah6gaMb1M66_bT565Mobbqj7B4nO0DQUQl5TtUtWeCnTDMRhqJ_lQxjyKMniBIv9-pnGBpQrh459XP68bTkA5bjZKxtq3mkjbPbDfn02eCKujTL5D5JXjGRf-b-X-TT8LRj85-5ojRIk-PnVepKwQ4nZKxtqtJcmBU76LR58OITgXq56jq0I3h34Bp3nWnFLaJQ8JfoDe4Tw5pD5KpTbyUn405OTfIFO0KJcb66vSR46hPJvyUAsXnO7Lx7lXbrtXp7_2J0WStbKy4oTjxL1Db3JKjvMtgDtVJO-KKC-MKPljM5; ZD_ENTRY=empty; BDRCVFR[dG2JNJb_ajR]=mk3SLVN4HKm; Hm_lvt_f06186a102b11eb5f7bcfeeab8d86b34=1617097541,1617155862,1617166647; Hm_lpvt_f06186a102b11eb5f7bcfeeab8d86b34=1617166647; Hm_lvt_d8bfb560f8d03bbefc9bdecafc4a4bf6=1617091648,1617094299,1617151626,1617166647; session_id=1617166647865; session_name=zhidao.baidu.com; LoseUserAllPage=%7B%22type%22%3A0%2C%22status%22%3A0%2C%22expire_time%22%3A0%2C%22create_time%22%3A1617166640%2C%22cookie_time%22%3A1617253040%7D; Hm_lpvt_d8bfb560f8d03bbefc9bdecafc4a4bf6=1617171461; ab_sr=1.0.0_MzRmYjc5YTgxYjM3MzMxY2M4ZWZkM2NhZWM1M2Q3YzM5NjkzNmM4OWU4NTU2OTE3OTEzODI4ODk4NzIzMjllN2QzNjQyMmZiOTAxODA1MGMwMWMwNWI2NTYxMzhjMTA5; bcat=6bc4acfc582093130d284bebe64dba2ceac7863f03574d982ec152edf0ad3e3161e522383677c411eaa9373fcbabcd2752cdca5d0c512d3483402be0384d65977066b234f8a33af4ec0620873490ed9a33834182c28b8295b7c08c01ed2ff8db5001b873f06ca2fb943614629ff44d2d345a27e0dcec081e49112f333a8580d4; ___wk_scode_token=gOwqyf8191hmzANHwpWgXqJHMYOvdAxj0G8Fg2cKaS0%3D",
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
}

client = MongoClient('mongodb://root:MgXwM3Toe5@192.168.7.53:30228')
baiduwenku_url_ = client["webpage"]["baiduwenku_url_"]
proxies = client["proxy"]["proxies_ip_dongtai"]
my_client = MongoClient()
my_coll = my_client["t"]["t3"]

ip = "dt"
proxies_list = [  # 静态代理列表
    "http://pc0308wd:pc0308wd@140.246.91.53:888",
    "http://pc0308wd:pc0308wd@113.125.125.2:888",
    "http://pc0308wd:pc0308wd@203.57.232.249:888",
    "http://pc0308wd:pc0308wd@113.125.9.8:888",
    "http://pc0308wd:pc0308wd@42.123.125.53:888",
]


# 使用动态代理
def get_ip_dt():
    items = list(proxies.find({}))
    item = random.choice(items)
    ip = item.get("ip")
    new_proxy = {
        "http": ip,
    }
    print(f"已设置代理 {ip}")
    return new_proxy


# 使用静态代理
def get_ip_jt():
    ip = random.choice(proxies_list)
    new_proxy = {
        "http": ip,
    }
    print(f"已设置代理 {ip}")
    return new_proxy


# 选择使用 静态代理 或者 动态代理
def get_ip():
    if ip == "jt":
        return get_ip_jt()
    elif ip == "dt":
        return get_ip_dt()


def save_db(item):
    baiduwenku_url_.insert_one(
        item
    )


def parse(url, params=None):
    if params == None:
        try:
            resp = requests.get(url, headers=headers, timeout=5, proxies=get_ip())
            return resp
        except Exception as e:
            print(e)
            return None
    else:
        try:
            resp = requests.get(url, headers=headers, params=params, timeout=5, proxies=get_ip())
            return resp
        except Exception as e:
            print(e)
            return None


def start(cid1_value):
    try:
        print("in start")
        url = f"https://wenku.baidu.com/portal/interface/getfiltertags?cid1={cid1_value}"
        resp = parse(url)
        print(resp)
        json_data = resp.json()
        tags = json_data["data"]["tags"]
        if not tags:
            print(">_<")
        # for tag in tags[:1]:
        for tag in tags:
            print(tag)
            l = tag["list"]
            # 小分类
            # for small_class in l[4:5]:
            for small_class in l:
                print(small_class)
                # 处理小分类
                parse_small_class(cid1_value, small_class)
                time.sleep(0.5)
    except Exception as e:
        print(e)


# 处理小分类
def parse_small_class(cid1_value, small_class):
    try:
        print("in parse_small_class")
        url_ = "https://wenku.baidu.com/portal/interface/getfilterresult?"

        check_id = None
        for pn_value in range(78):
            params = {
                'cid1': cid1_value,
                'tag': small_class,
                'rn': '24',
                'pn': pn_value,
            }
            # resp = requests.get(url_, params=params, headers=headers, proxies=get_ip())
            resp = parse(url_, params)
            if resp == None:
                continue
            print(f"正在采集第{pn_value + 1}页", small_class, resp.url)
            json_data = resp.json()
            info_list = json_data["data"]["docList"]

            if info_list == []:
                time.sleep(0.5)
                print(cid1_value, small_class, pn_value + 1, "不存在")
                print("正在重试")
                resp = parse(url_, params)
                if resp:
                    json_data = resp.json()
                    info_list = json_data["data"]["docList"]

                    if info_list == []:
                        print("重试失败")
                        continue
                    else:
                        print("重试成功")

                else:
                    print("重试失败")

            time.sleep(0.5)

            for info in info_list:
                id = info["docId"]
                if id == check_id:
                    print(f"只有{pn_value - 1}页", url_)
                    return
                check_id = id
                title = info["docTitle"]
                url = "https://wenku.baidu.com/view/" + id

                print(title)
                print(url)
                print()

                item = {}
                item["_id"] = encry_id(url)
                item["url"] = url

                # save(item)
                save_db(item)


    except Exception as e:
        print(e)


def save(item):
    my_coll.insert_one(item)


m = md5()


def encry_id(id):
    m.update(id.encode())
    return m.hexdigest()

    # url = ""
    # resp = parse(url)
    # json_data = resp.json()
    # info_list = json_data["data"]["docList"]
    # for info in info_list:
    #     id = info["docId"]
    #     title = info["docTitle"]
    #     print(title)
    #     print("https://wenku.baidu.com/view/" + id)
    #     print()


def main():
    # 12个板块
    # for cid1_value in range(1, 12):
    #     try:
    #         start(cid1_value)
    #     except Exception as e:
    #         try:
    #             start(cid1_value)
    #         except:
    #             pass
    #         print(e)

    # 先测试第一个模块
    # return

    with ProcessPoolExecutor(max_workers=11) as pool:
        for cid1_value in range(12):
            try:
                pool.submit(start, cid1_value)
            except Exception as e:
                print(e)


if __name__ == '__main__':
    for j in range(100):
        try:
            main()
        except:
            pass
